/**
 *
 * Copyright Pact Lab of H.I.T.
 *
 * Designed and Implemented by Grid Researching Group, 
 * Pact Lab, Harbin
 * 
 * This Project is part of the national 973 Project:
 * Internet Based Virtual Computing Environment
 *
 * http://pact518.hit.edu.cn
 * 
 * Author:       Meteor <meteorlxk@gmail.com> 
 * Copyright:    pact518 
 * Version:      1.0
 * Created:      2008-12-15 
 * LastModified: 2008-12-15
 */
package edu.hit.pact.pgse.crawler.util;

import java.net.MalformedURLException;
import java.util.*;

import edu.hit.pact.pgse.crawler.web.UrlUnit;

/**
 * @author Meteor
 *
 */
public class RobotFileManager {
	private Map<String, List<String>> robotsMap;
	
	private RobotFileScanner scanner;
	
	public RobotFileManager(RobotFileScanner scanner) {
		this.robotsMap = new HashMap<String, List<String>>();
		
		this.scanner = scanner;
	}
	
	public boolean disallow(UrlUnit url){
		List<String> robotsList;
		
		synchronized(this.robotsMap) {
			String hostName = url.getFullHostName();
			
			boolean containKey = robotsMap.containsKey(hostName);
			
			if (!containKey) {
				robotsList = scanner.scan(hostName);

				if (robotsList.isEmpty()) {
					robotsList = null;
				}//end if
				
 				robotsMap.put(hostName, robotsList);
			} else {
				robotsList = robotsMap.get(hostName);
			}
		}
		
		if (robotsList != null) {
			for (String disallowPath : robotsList) {
				if (url.getPath().startsWith(disallowPath)) {
					return true;
				}//end if
			}//end for
		}//end if
		
		return false;
	}
	
}
